package com.incito.crawl.common.utils.proxy;

import com.incito.crawl.common.utils.http.CrawlHttpClientConfig;
import com.incito.crawl.common.utils.http.CrawlerKit;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.codehaus.jackson.map.ObjectMapper;

import java.io.IOException;
import java.util.*;
import java.util.concurrent.ArrayBlockingQueue;

/**
 * Created by IntelliJ IDEA.
 * User: YZC
 * Date: 12-11-8
 * Time: 下午1:30
 * To change this template use File | Settings | File Templates.
 */
public class ProxyKit {
    private static org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProxyKit.class);
    public static final String defaultRuleName = "s-baidu";
    private static final String baseProxyUrl = CrawlHttpClientConfig.getBaseProxyUrl();
    private static Map<String, Queue<ProxyInfo>> proxyMap = new HashMap<String, Queue<ProxyInfo>>();
    private static Map<String, ProxyCacheBean> proxyBeanCache = new HashMap<String, ProxyCacheBean>();

    //代理规则下的代理上次刷新时间的Map
    private static Map<String, Long> lastRefreshTimeMap = new HashMap<String, Long>();

    //默认300秒
    private static final int minRefreshSeconds = 300;

    private static RequestConfig.Builder requestConfigBuilder = CrawlerKit.createRequestConfigBuilder(false, 3000, 5000, 10);

    private static int proxyQueueSize = 1000;

    static {
        requestConfigBuilder.setConnectTimeout(5000);
        requestConfigBuilder.setSocketTimeout(5000);
    }

    private static RequestConfig requestConfig = requestConfigBuilder.build();

    private static ObjectMapper objectMapper = new ObjectMapper();

    /**
     * lastRefreshTime 方面判断 是否应该刷新内存中的所有proxyInfo，即重新请求代理
     *
     * @param ruleName
     * @param curTime
     * @return 返回true则需要刷新
     */
    public static boolean isRefreshNow(String ruleName, long curTime) {
        boolean refreshFlag = false;
        Long lastRefreshTime = lastRefreshTimeMap.get(ruleName);
        if (lastRefreshTime != null && lastRefreshTime > 0) {
            refreshFlag = (curTime - lastRefreshTime) / 1000 > minRefreshSeconds;
        } else {
            refreshFlag = true;
        }
        return refreshFlag;
    }

    public static synchronized ProxyInfo getProxy() {
        return getProxy(defaultRuleName);
    }

    public static synchronized ProxyInfo getProxy(String ruleName) {
        long curTime = System.currentTimeMillis();
        Queue<ProxyInfo> proxyQueue = getProxyQueueFromProxyMap(ruleName);

        int queueSize = proxyQueue.size();
        if (proxyQueue.isEmpty() || queueSize <= 30 || isRefreshNow(ruleName, curTime)) {
            ProxyInfoData proxyInfoData = loadProxies(ruleName);
            if (proxyInfoData != null) {
                List<ProxyInfo> proxyList = Arrays.asList(proxyInfoData.getResult());
                Collections.shuffle(proxyList);
                addProxyInfosToProxyQueue(proxyQueue, proxyList);
            }

            if (proxyQueue.size() <= 30 && !defaultRuleName.equals(ruleName)) {
                ProxyInfoData defaultProxyInfoData = loadProxies(defaultRuleName);
                if (proxyInfoData != null) {
                    List<ProxyInfo> defaultProxyList = Arrays.asList(defaultProxyInfoData.getResult());
                    Collections.shuffle(defaultProxyList);
                    addProxyInfosToProxyQueue(proxyQueue, defaultProxyList);
                }
            }
            logger.info("load [{}] queue size : [{}]", ruleName, proxyQueue.size());
        }

        ProxyInfo proxyInfo = null;
        if (proxyQueue.size() > 0)
            proxyInfo = proxyQueue.poll();

        if (proxyInfo == null) {
            logger.debug(" --------    proxy queue size :  {}", proxyQueue.size());
        }
        return proxyInfo;
    }

    private static Queue<ProxyInfo> getProxyQueueFromProxyMap(String proxyRuleName) {
        Queue<ProxyInfo> proxyQueue = null;
        proxyQueue = proxyMap.get(proxyRuleName);
        if (proxyQueue == null) {
            proxyQueue = new ArrayBlockingQueue<ProxyInfo>(proxyQueueSize);
            proxyMap.put(proxyRuleName, proxyQueue);
        }
        return proxyQueue;
    }

    /**
     * 加载代理信息
     *
     * @param ruleName
     */
    private static ProxyInfoData loadProxies(String ruleName) {
        long curTime = System.currentTimeMillis();
        ProxyInfoData proxyInfoData = null;
        ProxyInfoData cacheProxyInfoData = null;

        ProxyCacheBean proxyCacheBean = null;


        //获取缓存中的代理信息
        if (proxyBeanCache.containsKey(ruleName)) {
            proxyCacheBean = proxyBeanCache.get(ruleName);
            cacheProxyInfoData = proxyCacheBean.data;
        }

        if (cacheProxyInfoData != null && !isRefreshNow(ruleName, curTime) && cacheProxyInfoData.getResult().length > 30) {
            //存在对应规则的代理缓存信息同时没有到达刷新时间，则直接从缓存加载代理信息
            logger.info("代理队列:{} 被耗完且还未到达刷新时间（5分钟）,从缓存中重新装载", ruleName);
            proxyInfoData = cacheProxyInfoData;
        } else {
            //从代理信息api加载代理信息并更新缓存信息
            ProxyInfoData pdata = loadProxiesFromApi(ruleName);
            if (pdata != null && pdata.getResult().length > 0) {
                proxyInfoData = pdata;
            } else {
                proxyInfoData = cacheProxyInfoData;
            }
        }
        return proxyInfoData;
    }

    /**
     * 从api获取代理信息,同时按ruleName刷新代理的缓存和代理的Map
     *
     * @param ruleName
     * @return
     */
    private static ProxyInfoData loadProxiesFromApi(String ruleName) {
        ProxyInfoData proxyInfoData = null;
        String proxyUrl = baseProxyUrl  + ruleName;
        logger.info("从Api加载规则 {} 的代理信息，proxyUrl:" + proxyUrl, ruleName);
        CloseableHttpClient httpClient = HttpClientBuilder.create().setDefaultRequestConfig(requestConfig).build();
        CloseableHttpResponse httpResponse = null;

        try {

            HttpGet req = new HttpGet(proxyUrl);
            RequestConfig requestConfig = CrawlerKit.createRequestConfigBuilder(false, 3000, 8000, 1).build();
            req.setConfig(requestConfig);

            HttpContext httpClientContext = new HttpClientContext();
            httpResponse = httpClient.execute(req, httpClientContext);

            logger.info(" @@@@@@@@@@@@@@@@@ Get : " + proxyUrl);
            if (httpResponse.getStatusLine().getStatusCode() != 200) {
                Thread.sleep(10000);
                httpResponse = httpClient.execute(req, httpClientContext);
                logger.info(" @@@@@@@@@@@@@@@@@ Get : " + proxyUrl);
            }
            if (httpResponse.getStatusLine().getStatusCode() == 200) {
                try {
                    String html = EntityUtils.toString(httpResponse.getEntity());
                    proxyInfoData = objectMapper.readValue(html, ProxyInfoData.class);
                } catch (Exception e) {
                    logger.error("分析代理异常：{}", e);
                }
            }

        } catch (Exception e) {
            logger.error("获取代理异常：{}", e);
        } finally {
            try {
                if (httpResponse != null) {
                    httpResponse.close();
                }
            } catch (Exception e) {
                //no hup......
            }

            try {
                httpClient.close();
            } catch (IOException e) {
                //no hup......
            }
        }

        if (proxyInfoData != null) {
            long curTime = System.currentTimeMillis();
            lastRefreshTimeMap.put(ruleName, curTime);
            ProxyCacheBean proxyCacheBean = new ProxyCacheBean();
            proxyCacheBean.data = proxyInfoData;
            proxyBeanCache.put(ruleName, proxyCacheBean);
        }

        return proxyInfoData;
    }

    private static void addProxyInfosToProxyQueue(Queue<ProxyInfo> proxyQueue, Collection<ProxyInfo> proxyInfos) {
        if (proxyQueue == null || proxyInfos == null) return;
//        Set<ProxyInfo> proxyInfoSet = new HashSet<ProxyInfo>();
//        proxyInfoSet.addAll(proxyQueue);
        for (ProxyInfo proxyInfo : proxyInfos) {
//            if (proxyInfoSet.add(proxyInfo)) {
            if (proxyQueue.size() < proxyQueueSize && !proxyQueue.contains(proxyInfo)) {
                proxyQueue.add(proxyInfo);
            } else {
                continue;
            }
//            }
        }
    }

    private static void addProxyInfoToProxyQueue(Queue<ProxyInfo> proxyQueue, ProxyInfo proxyInfo) {
        if (proxyQueue == null || proxyInfo == null) return;
//        Set<ProxyInfo> proxyInfoSet = new HashSet<ProxyInfo>();
//        proxyInfoSet.addAll(proxyQueue);
//        if (proxyInfoSet.add(proxyInfo)) {
//            if (proxyQueue.size() < proxyQueueSize) {
//                proxyQueue.add(proxyInfo);
//            }
//        }
        if (proxyQueue.size() < proxyQueueSize && !proxyQueue.contains(proxyInfo)) {
            proxyQueue.add(proxyInfo);
        }
    }

    /**
     * 将代理回写到队列中，重复利用
     *
     * @param proxyInfo
     * @param proxyRuleName
     */
    public static void returnProxyInfoToProxyQueue(ProxyInfo proxyInfo, String proxyRuleName) {
        String proxyName = proxyRuleName;
        if (StringUtils.isBlank(proxyName))
            proxyName = defaultRuleName;
        try {
            Queue<ProxyInfo> proxyQueue = proxyMap.get(proxyName);
            addProxyInfoToProxyQueue(proxyQueue, proxyInfo);
            logger.info("返回可用代理成功{}", proxyInfo.toString());
        } catch (Exception e) {
            logger.error("返回可用代理异常：{}", e);
        }
    }

    public static void main(String args[]) {
        int i = 0;
        while (i < 200) {
            System.out.println(ProxyKit.getProxy("s-baidu"));
            i++;
        }
    }
}
